import pandas as pd, numpy as np, matplotlib.pyplot as plt, matplotlib
import hvplot.pandas
import tensorflow as tf
from termcolor import colored
pd.options.plotting.backend = 'holoviews'
matplotlib.style.use("seaborn-whitegrid")
pd.set_option("display.width", 5000)
pd.set_option("display.max_columns", 60)
plt.rcParams["figure.figsize"] = (15, 10)
import tensorflow.keras as keras
import tensorflow_datasets as tfds
from tensorboard.plugins import projector
%config InlineBackend.figure_format = 'retina'
# %config InlineBackend.figure_format = 'svg'
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
import warnings
warnings.simplefilter('ignore')
pd.set_option('display.float_format', lambda x: '%.3f' % x)
df = pd.read_csv(
"https://raw.githubusercontent.com/t-davidson/hate-speech-and-offensive-language/master/data/labeled_data.csv"
)
positive = pd.read_csv('/Users/glebsokolov/Downloads/archive/processedPositive.csv').T.reset_index().rename(columns={'index':'tweet'})
neutral = pd.read_csv('/Users/glebsokolov/Downloads/archive/processedNeutral.csv').T.reset_index().rename(columns={'index':'tweet'})
positive['class'], neutral['class'] = 2, 2
positive.head();
neutral.head();
count = number of CrowdFlower users who coded each tweet (min is 3, sometimes more users coded a tweet when judgments were determined to be unreliable by CF).
hate_speech = number of CF users who judged the tweet to be hate speech.
offensive_language = number of CF users who judged the tweet to be offensive.
neither = number of CF users who judged the tweet to be neither offensive nor non-offensive.
class = class label for majority of CF users. 0 - hate speech 1 - offensive language 2 - neither
df.info()
# df.drop('Unnamed: 0', axis=1, inplace=True)
df.head()
df['class'].value_counts(normalize=True)
df = pd.concat([df[['tweet', 'class']], positive, neutral], axis=0, ignore_index=True)
df['class'].value_counts(normalize=True)
<class 'pandas.core.frame.DataFrame'> RangeIndex: 24783 entries, 0 to 24782 Data columns (total 7 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 Unnamed: 0 24783 non-null int64 1 count 24783 non-null int64 2 hate_speech 24783 non-null int64 3 offensive_language 24783 non-null int64 4 neither 24783 non-null int64 5 class 24783 non-null int64 6 tweet 24783 non-null object dtypes: int64(6), object(1) memory usage: 1.3+ MB
| Unnamed: 0 | count | hate_speech | offensive_language | neither | class | tweet | |
|---|---|---|---|---|---|---|---|
| 0 | 0 | 3 | 0 | 0 | 3 | 2 | !!! RT @mayasolovely: As a woman you shouldn't... |
| 1 | 1 | 3 | 0 | 3 | 0 | 1 | !!!!! RT @mleew17: boy dats cold...tyga dwn ba... |
| 2 | 2 | 3 | 0 | 3 | 0 | 1 | !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby... |
| 3 | 3 | 3 | 0 | 2 | 1 | 1 | !!!!!!!!! RT @C_G_Anderson: @viva_based she lo... |
| 4 | 4 | 6 | 0 | 6 | 0 | 1 | !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you... |
1 0.774 2 0.168 0 0.058 Name: class, dtype: float64
1 0.697 2 0.251 0 0.052 Name: class, dtype: float64
import os
directory = '/Users/glebsokolov/hate-speech-dataset/all_files/class_a/'
files = os.listdir('/Users/glebsokolov/hate-speech-dataset/all_files/class_a/')
i = df.shape[0]
hatespeechlist=[]
for file in files:
i += 1
with open(os.path.join(directory, file), 'r') as f:
text = f.read()
hatespeechlist.append(text)
temp = pd.DataFrame([{'tweet':text, 'class':0} for text in hatespeechlist], index=list(range(len(hatespeechlist))))
df = pd.concat([df, temp], ignore_index=True)
df['class'].value_counts(normalize=True)
1 0.499 0 0.322 2 0.180 Name: class, dtype: float64
df = df.sample(frac=1)
train = df.sample(frac=0.8)
test = df.drop(labels=train.index)
def preprocess(X_batch, y_batch):
X_batch = tf.strings.substr(X_batch, 0, 300)
X_batch = tf.strings.lower(X_batch)
X_batch = tf.strings.regex_replace(X_batch, b"<br\\s*/?>", b" ")
X_batch = tf.strings.regex_replace(X_batch, b"[^a-zA-Z']", b" ")
X_batch = tf.strings.split(X_batch)
return X_batch.to_tensor(default_value=b"<pad>"), y_batch
texts = tf.data.Dataset.from_tensor_slices(preprocess(train.tweet, train["class"])[0])
test_texts = tf.data.Dataset.from_tensor_slices(
preprocess(test.tweet, test["class"])[0]
)
2022-05-26 17:19:26.080627: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
from keras.utils.np_utils import to_categorical
y, test_y = tf.data.Dataset.from_tensor_slices(
to_categorical(train["class"])
), tf.data.Dataset.from_tensor_slices(to_categorical(test["class"]))
train, test = tf.data.Dataset.zip((texts, y)), tf.data.Dataset.zip((test_texts, test_y))
from collections import Counter
vocabulary = Counter()
for X_batch, y_batch in train.batch(32):
for review in X_batch:
vocabulary.update(list(review.numpy()))
vocab_size = 10000
truncated_vocabulary = [word for word, count in vocabulary.most_common()[:vocab_size]]
words = tf.constant(truncated_vocabulary)
word_ids = tf.range(len(truncated_vocabulary), dtype=tf.int64)
vocab_init = tf.lookup.KeyValueTensorInitializer(words, word_ids)
num_oov_buckets = 1000
table = tf.lookup.StaticVocabularyTable(vocab_init, num_oov_buckets)
def encode_words(X_batch, y_batch):
return table.lookup(X_batch), y_batch
train, test = train.batch(32).map(encode_words).prefetch(1), test.batch(32).map(
encode_words
).prefetch(1)
(train_data, test_data), info = tfds.load(
"imdb_reviews/subwords8k",
split=(tfds.Split.TRAIN, tfds.Split.TEST),
with_info=True,
as_supervised=True,
)
encoder = info.features["text"].encoder
WARNING:absl:TFDS datasets with text encoding are deprecated and will be removed in a future version. Instead, you should use the plain text version and tokenize the text using `tensorflow_text` (See: https://www.tensorflow.org/tutorials/tensorflow_text/intro#tfdata_example) 2022-05-26 11:03:40.514766: W tensorflow/core/platform/cloud/google_auth_provider.cc:184] All attempts to get a Google authentication bearer token failed, returning an empty token. Retrieving token from files failed with "NOT_FOUND: Could not locate the credentials file.". Retrieving token from GCE failed with "FAILED_PRECONDITION: Error executing an HTTP request: libcurl code 6 meaning 'Couldn't resolve host name', error details: Could not resolve host: metadata".
Downloading and preparing dataset 80.23 MiB (download: 80.23 MiB, generated: Unknown size, total: 80.23 MiB) to /Users/glebsokolov/tensorflow_datasets/imdb_reviews/subwords8k/1.0.0...
Dl Completed...: 0 url [00:00, ? url/s]
Dl Size...: 0 MiB [00:00, ? MiB/s]
Generating splits...: 0%| | 0/3 [00:00<?, ? splits/s]
Generating train examples...: 0%| | 0/25000 [00:00<?, ? examples/s]
Shuffling /Users/glebsokolov/tensorflow_datasets/imdb_reviews/subwords8k/1.0.0.incomplete0IORLZ/imdb_reviews-t…
Generating test examples...: 0%| | 0/25000 [00:00<?, ? examples/s]
Shuffling /Users/glebsokolov/tensorflow_datasets/imdb_reviews/subwords8k/1.0.0.incomplete0IORLZ/imdb_reviews-t…
Generating unsupervised examples...: 0%| | 0/50000 [00:00<?, ? examples/s]
Shuffling /Users/glebsokolov/tensorflow_datasets/imdb_reviews/subwords8k/1.0.0.incomplete0IORLZ/imdb_reviews-u…
WARNING:absl:Dataset is using deprecated text encoder API which will be removed soon. Please use the plain_text version of the dataset and migrate to `tensorflow_text`.
Dataset imdb_reviews downloaded and prepared to /Users/glebsokolov/tensorflow_datasets/imdb_reviews/subwords8k/1.0.0. Subsequent calls will reuse this data.
import os
# Set up a logs directory, so Tensorboard knows where to look for files.
log_dir=os.path.join(os.getcwd(),'logs/hateSpeechDetection/')
if not os.path.exists(log_dir):
os.makedirs(log_dir)
%%capture
# Save Labels separately on a line-by-line manner.
with open(os.path.join(log_dir, 'metadata.tsv'), "w") as f:
for word in words:
f.write("{}\n".format(word))
# Fill in the rest of the labels with "unknown".
for unknown in range(1, num_oov_buckets):
f.write("unknown #{}\n".format(unknown))
from sklearn.decomposition import PCA
embed_size = 128
embedding = keras.layers.Embedding(
vocab_size + num_oov_buckets, embed_size, input_shape=[None], mask_zero=True
)
model = keras.models.Sequential(
[
embedding,
keras.layers.GRU(128, return_sequences=True),
keras.layers.GRU(128),
keras.layers.Dense(3, activation="softmax"),
]
)
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
early_stopping_cb = keras.callbacks.EarlyStopping(patience=2, restore_best_weights=True)
history = model.fit(
train, epochs=5, validation_data=test, callbacks=early_stopping_cb
) # disabled after learning
Epoch 1/5 963/963 [==============================] - 182s 181ms/step - loss: 0.3778 - accuracy: 0.8582 - val_loss: 0.2777 - val_accuracy: 0.9068 Epoch 2/5 963/963 [==============================] - 167s 173ms/step - loss: 0.2130 - accuracy: 0.9273 - val_loss: 0.2920 - val_accuracy: 0.9019 Epoch 3/5 963/963 [==============================] - 168s 175ms/step - loss: 0.1419 - accuracy: 0.9539 - val_loss: 0.3776 - val_accuracy: 0.8918
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = "embedding/.ATTRIBUTES/VARIABLE_VALUE"
embedding.metadata_path = 'metadata.tsv'
projector.visualize_embeddings(log_dir, config)
from sklearn.manifold import LocallyLinearEmbedding
pca, lle = PCA(n_components=2), LocallyLinearEmbedding(n_components=2)
weights = tf.Variable(model.layers[0].get_weights()[0][1:])
reduced_pca, reduced_lle = pca.fit_transform(weights), lle.fit_transform(weights)
import os
with open(os.path.join(os.getcwd(), 'metadata.tsv'), "w") as f:
for word in words:
f.write("{}\n".format(word))
9
7
5
5
6
7
9
7
6
6
6
8
6
6
7
6
6
8
8
6
6
5
8
8
11
6
8
7
8
7
8
7
6
8
8
9
7
6
6
7
7
7
6
6
8
7
7
9
6
6
7
9
6
9
9
7
6
7
5
8
7
7
8
8
8
6
8
8
7
9
7
9
7
8
8
7
9
9
6
10
9
8
6
7
8
9
9
8
6
7
7
8
6
8
8
7
10
7
8
7
8
9
7
8
8
7
8
7
7
7
7
8
9
7
8
8
9
6
6
8
9
7
9
8
7
11
11
7
8
8
10
9
9
9
10
8
6
11
7
8
10
10
8
7
9
9
8
5
8
8
7
8
6
9
6
8
6
10
5
9
7
5
10
8
10
7
8
7
8
10
8
8
8
7
9
10
8
8
9
8
9
8
9
8
8
10
8
7
9
8
7
8
9
10
7
10
8
8
9
11
8
6
9
10
9
8
8
5
8
5
9
8
7
8
6
7
9
8
11
11
7
11
9
9
7
9
9
13
7
7
8
11
10
11
11
8
8
7
8
10
8
8
8
8
8
10
8
8
10
9
9
11
8
7
7
8
6
9
7
8
8
9
6
9
8
9
8
11
8
8
7
8
9
7
9
7
8
12
8
8
10
8
10
8
7
10
8
7
10
9
5
10
5
7
7
5
8
5
9
12
9
8
10
8
9
8
8
8
6
8
8
11
10
9
6
8
7
8
6
8
10
8
7
7
8
8
11
9
10
7
7
8
8
9
8
10
5
5
7
7
8
9
9
11
12
9
11
8
8
8
9
8
8
8
9
10
8
10
9
10
7
10
9
9
9
10
8
8
9
8
9
7
8
5
7
7
8
8
9
9
11
9
8
11
10
8
7
9
8
12
8
9
7
8
5
12
8
9
9
9
8
8
8
6
8
8
8
9
7
7
8
7
5
9
7
10
9
14
7
7
9
8
10
9
9
12
5
9
9
10
7
7
10
9
5
7
8
11
8
9
10
8
11
7
9
5
11
8
10
11
8
7
11
9
6
8
9
9
5
10
8
8
7
8
7
5
12
12
8
10
8
7
12
7
7
9
12
6
9
7
8
8
9
6
7
8
13
6
10
8
11
9
8
8
5
7
7
8
9
9
14
8
7
7
9
7
10
5
8
10
10
10
8
7
8
12
13
8
8
6
10
9
10
8
8
9
10
7
8
7
10
9
12
8
10
11
6
8
9
11
9
8
10
7
10
11
8
9
11
10
8
10
9
10
7
8
8
8
8
11
8
7
8
9
13
7
8
8
9
10
7
7
10
6
9
9
11
7
9
9
9
9
9
10
11
6
11
9
9
11
8
12
9
8
8
9
10
9
8
8
8
10
8
7
8
10
12
11
7
10
11
7
10
7
8
8
10
9
9
11
10
11
11
8
9
11
9
9
11
7
8
9
12
6
11
9
9
9
10
8
9
9
9
11
9
11
8
7
7
12
8
9
8
14
7
7
11
7
10
13
9
6
9
9
9
10
6
14
10
8
12
10
6
7
7
11
12
8
9
9
11
11
10
7
10
8
11
7
9
8
6
11
14
7
9
7
8
10
9
10
11
5
10
9
10
11
9
10
15
8
10
11
10
9
8
9
8
8
9
8
9
10
11
11
9
11
7
9
12
7
9
9
9
12
10
8
8
8
6
9
7
8
9
8
7
7
12
9
8
8
8
8
6
8
8
9
10
8
9
9
8
8
13
11
8
8
6
9
8
9
10
8
12
9
8
12
6
11
9
8
10
13
8
9
8
9
12
14
11
8
10
11
9
8
9
10
8
12
10
12
7
12
9
8
12
9
8
10
10
8
10
10
9
7
5
9
11
7
13
9
10
12
8
11
10
13
10
9
8
9
8
10
9
8
7
9
9
8
7
7
7
11
8
10
8
7
10
13
8
10
8
10
9
7
9
11
9
6
10
11
8
8
11
10
8
8
10
9
10
10
8
7
11
9
7
10
9
9
11
11
7
8
8
12
9
13
6
8
10
8
13
8
10
8
10
11
9
9
9
10
8
10
8
10
12
9
9
12
10
9
9
6
14
10
12
9
7
9
8
11
8
7
8
12
7
10
12
10
9
9
10
10
10
9
8
10
11
16
8
10
8
6
9
8
11
11
7
13
7
9
10
11
11
9
10
9
11
8
9
7
8
9
8
10
9
12
6
8
12
11
7
10
10
10
9
8
8
10
10
9
11
12
9
12
8
8
8
6
8
11
9
8
12
10
10
11
10
9
9
8
12
8
14
7
8
8
8
8
7
10
7
10
7
7
9
9
9
8
10
10
8
9
9
8
9
8
7
7
9
10
8
10
9
11
9
9
8
9
7
10
11
10
11
12
9
9
8
8
11
12
8
11
15
10
13
6
11
10
9
8
9
9
9
10
9
9
9
8
11
11
9
8
11
13
8
13
8
8
10
7
10
9
10
9
8
11
7
8
11
10
9
6
8
9
8
9
8
9
11
11
8
7
8
13
14
8
11
7
9
9
9
8
9
11
8
11
7
13
7
10
6
11
11
14
10
8
9
9
9
9
9
13
9
9
10
8
11
8
8
10
9
9
14
9
9
11
12
11
7
7
9
10
8
13
10
10
11
8
8
9
11
9
13
7
8
11
9
10
10
8
11
10
8
9
13
8
9
8
9
10
7
8
9
10
12
9
10
6
9
8
8
10
8
11
13
9
8
14
9
7
13
10
11
11
6
11
12
11
9
8
10
9
15
9
14
7
9
10
9
7
6
9
9
8
10
9
9
10
9
8
9
11
8
10
10
9
8
12
11
13
11
8
9
9
11
9
9
8
8
11
10
8
8
10
9
9
7
9
12
10
11
11
12
9
11
11
7
8
9
9
6
12
7
11
12
7
8
7
13
11
9
9
11
9
8
11
9
7
12
6
10
10
8
6
6
9
7
7
11
9
18
10
6
12
8
13
9
11
12
10
9
10
8
8
11
12
7
14
8
11
8
15
8
8
6
7
12
8
9
7
9
10
8
12
7
8
9
10
9
9
9
10
6
7
11
12
10
7
10
8
9
9
9
14
8
8
9
11
8
8
6
9
7
6
9
14
13
11
8
9
9
7
9
10
10
12
10
10
9
8
10
9
9
8
8
10
9
11
12
11
11
9
8
10
10
14
8
6
9
8
11
11
10
11
14
9
11
6
8
14
10
9
7
8
8
11
9
8
10
8
7
11
8
11
8
10
11
8
10
12
9
9
14
11
8
10
8
10
9
8
12
12
8
10
9
9
8
12
13
10
9
8
11
9
8
9
12
12
11
11
9
7
12
8
12
11
12
11
11
10
11
13
9
10
9
14
11
9
7
7
10
6
10
9
10
9
12
9
11
10
10
15
8
7
11
10
8
9
9
11
9
12
11
6
9
9
6
8
7
12
8
10
7
9
11
8
12
9
11
8
13
16
8
9
7
8
12
8
10
8
12
8
11
12
13
10
7
11
8
6
9
11
11
6
11
10
10
14
9
9
8
8
9
9
8
7
10
13
8
7
13
12
12
7
9
7
11
8
9
12
8
7
11
8
8
13
9
11
10
9
11
10
10
9
8
9
12
9
11
8
9
7
11
8
9
8
10
10
13
8
8
8
9
10
12
8
9
7
9
13
8
13
9
8
11
7
8
11
7
6
10
8
7
11
10
10
14
11
9
11
11
12
13
19
7
10
10
12
10
14
8
9
9
10
7
9
9
10
9
12
11
12
12
10
10
9
10
8
8
8
11
11
10
11
9
9
7
8
8
9
12
10
7
9
10
12
7
7
7
7
11
11
9
7
11
8
9
11
9
10
10
10
11
8
9
10
12
9
8
11
10
11
9
7
9
11
10
10
6
10
15
11
10
8
12
11
10
7
9
9
11
11
8
11
10
8
11
13
10
8
9
8
8
16
9
9
17
10
11
7
12
10
9
8
11
8
10
8
8
10
7
12
10
11
8
11
10
7
9
12
18
8
9
13
8
11
13
8
8
10
10
8
9
9
10
11
10
9
11
10
10
11
14
10
8
9
8
6
8
10
7
9
11
9
12
7
8
10
8
9
8
9
10
9
11
10
13
9
12
9
11
9
8
11
7
8
6
8
13
7
9
9
7
17
11
10
10
16
12
11
12
11
9
12
10
12
10
8
8
7
11
11
9
9
11
9
9
9
8
7
10
10
9
11
7
8
9
8
8
10
14
11
18
12
8
10
11
9
8
7
14
11
8
9
10
11
9
10
9
8
10
9
9
9
9
10
9
10
12
8
8
8
17
17
7
8
7
11
12
10
9
10
9
7
8
6
8
11
6
9
10
9
10
7
10
10
10
10
11
10
13
11
7
6
9
8
10
10
14
11
11
10
10
7
9
11
11
11
11
8
6
9
7
8
8
9
9
8
10
6
9
16
9
9
10
15
12
11
11
10
8
11
10
12
8
11
13
10
7
9
10
9
7
13
9
12
7
11
9
9
9
12
8
10
9
9
9
10
11
11
10
10
6
10
10
9
10
12
10
7
10
9
7
7
9
12
9
9
8
10
10
10
10
10
11
12
10
7
10
8
10
6
10
11
6
7
9
10
9
12
11
7
10
9
8
9
9
9
13
10
11
8
10
9
6
9
9
7
13
9
13
14
9
11
11
9
10
15
10
7
10
11
8
11
7
15
8
10
11
10
10
7
8
11
7
7
8
9
11
12
6
10
10
11
14
12
12
9
11
6
7
9
6
10
10
10
9
7
13
10
6
11
10
8
10
10
14
9
11
12
10
6
12
7
6
8
8
10
9
6
12
9
9
10
9
7
10
12
9
8
9
15
12
11
11
9
13
9
9
8
10
11
8
10
16
9
7
10
6
10
11
9
9
11
15
10
9
10
8
10
12
9
11
11
9
11
8
7
10
6
10
8
8
9
9
10
8
12
10
10
10
8
9
11
8
6
10
8
9
14
12
10
9
11
9
10
8
11
7
6
10
9
8
12
10
10
10
7
8
10
9
11
12
12
11
7
10
6
13
8
11
11
9
10
11
9
7
14
11
7
12
9
15
8
7
9
10
10
12
11
14
11
12
9
9
11
8
8
7
8
8
9
12
13
12
10
12
8
11
14
7
13
12
11
11
12
10
9
14
10
10
7
16
8
12
9
13
11
10
10
7
7
10
11
8
7
6
11
11
7
7
8
10
8
11
12
10
9
7
10
12
10
12
11
9
10
10
9
10
12
10
11
9
12
12
13
8
10
11
12
6
11
8
11
8
13
9
11
13
10
12
12
11
9
12
10
8
9
12
10
11
10
7
11
11
16
9
15
6
9
11
9
9
8
9
16
10
10
8
11
9
9
9
9
9
9
12
10
10
10
9
9
9
9
13
9
12
7
9
7
16
10
11
8
9
12
7
11
12
12
7
11
11
12
10
8
8
9
11
7
8
10
6
9
10
9
12
7
10
12
12
10
9
15
13
11
8
9
9
9
8
15
7
11
7
13
10
9
7
8
11
16
7
14
9
19
12
8
10
11
6
11
20
8
11
6
7
10
9
9
10
10
10
10
9
10
8
11
8
8
10
12
8
12
13
13
11
17
9
14
7
6
12
9
12
7
7
8
9
11
6
13
10
8
11
12
9
8
10
9
11
12
11
8
10
11
12
11
11
16
12
9
6
6
8
9
11
10
9
11
7
8
9
10
9
6
7
9
9
9
11
12
10
10
9
10
12
10
11
12
8
9
8
9
10
8
11
11
9
13
13
14
13
17
10
11
9
12
11
6
10
7
8
10
12
8
11
10
8
9
10
11
10
9
7
10
9
6
12
9
11
6
11
10
14
6
10
10
11
6
10
12
8
11
10
11
9
11
9
9
11
11
9
11
9
7
10
10
19
11
9
8
9
12
10
11
8
13
8
9
7
7
8
10
10
8
12
9
13
9
10
9
9
11
9
7
12
9
10
14
8
7
11
10
12
13
9
10
8
8
9
10
11
10
12
11
7
9
11
11
7
12
11
11
12
12
7
10
12
8
16
10
6
9
13
9
8
12
13
9
10
8
11
9
10
10
14
10
8
8
8
13
16
14
11
10
7
13
9
8
7
6
12
9
10
8
12
10
10
15
10
10
10
10
9
10
8
7
10
10
8
9
13
10
9
8
9
9
9
13
11
7
12
11
9
10
10
10
12
11
10
8
13
9
9
16
7
11
6
9
7
8
8
9
12
11
9
7
13
11
8
9
9
8
12
10
8
8
10
13
11
7
10
10
9
10
9
9
10
6
11
10
11
10
9
6
11
7
12
11
10
14
9
11
12
7
7
6
11
8
8
10
10
7
7
12
11
8
15
12
9
10
7
16
8
10
12
15
8
8
14
10
8
12
8
9
9
6
7
9
19
7
12
8
12
7
10
8
11
11
11
14
9
12
6
17
10
10
12
10
8
8
12
13
6
8
11
12
8
9
12
11
8
11
6
14
12
7
9
6
6
11
13
11
11
12
10
10
9
11
9
9
8
8
8
11
11
9
8
11
8
10
11
13
9
12
6
10
11
12
10
10
12
10
13
7
13
11
10
9
7
10
9
11
7
9
8
10
8
7
9
9
8
9
8
7
13
9
6
9
12
9
11
11
11
10
8
8
12
11
10
15
11
9
10
17
8
8
10
11
9
9
8
8
9
10
12
14
9
7
9
6
11
11
9
11
11
11
11
9
11
7
9
7
10
8
10
8
11
10
7
8
7
11
9
17
9
13
9
9
11
12
9
17
10
8
12
9
9
9
9
11
14
11
9
17
10
9
9
8
8
13
8
6
8
11
9
8
10
14
12
7
10
6
6
8
9
9
7
7
9
11
11
12
10
6
9
9
11
10
8
8
10
11
10
10
6
14
9
8
11
11
7
10
9
9
7
15
12
11
12
9
8
7
10
9
11
10
9
11
13
6
11
11
10
9
10
10
11
10
8
12
12
9
13
9
9
12
6
6
9
9
13
8
12
12
6
13
8
8
11
10
13
9
9
6
11
7
11
10
12
7
11
9
6
9
10
8
11
7
10
12
7
9
12
12
8
9
11
13
8
10
14
11
11
6
8
9
18
9
8
6
9
10
11
12
13
9
12
10
10
10
10
9
9
9
7
9
9
9
10
11
12
9
13
7
10
11
7
12
9
9
7
11
12
11
9
13
11
12
11
7
10
10
7
9
10
13
7
11
9
9
11
10
10
7
7
7
7
9
12
11
8
8
10
10
9
12
11
9
8
10
9
7
6
10
9
13
8
11
9
13
7
9
12
11
15
13
11
11
15
15
7
11
13
9
10
11
8
6
8
9
9
15
12
8
11
9
11
15
14
7
13
9
7
10
13
7
9
8
8
9
13
8
10
6
10
7
8
8
12
12
11
12
9
8
8
8
8
7
7
9
13
11
9
8
11
10
9
15
8
11
8
9
9
9
6
10
13
8
8
10
8
10
14
10
10
9
6
8
9
11
10
12
13
8
11
10
14
10
9
10
8
9
12
11
12
8
11
10
11
10
10
10
14
11
9
8
11
10
7
6
9
8
8
9
13
8
11
11
9
12
10
9
12
11
12
9
8
7
9
10
6
6
9
9
10
8
13
11
9
9
10
10
10
12
8
6
9
18
10
11
11
10
9
13
11
9
9
9
11
13
11
7
9
14
11
10
11
6
10
10
10
12
8
10
7
8
12
7
14
9
7
9
12
9
11
6
13
12
10
9
8
11
11
9
7
14
14
13
9
10
11
8
8
10
12
10
8
6
13
8
11
13
9
12
12
9
11
8
10
12
9
9
8
10
11
10
7
8
9
9
10
7
9
9
10
9
10
14
8
19
10
12
8
14
11
17
7
13
11
12
8
14
12
7
10
10
11
11
10
14
10
13
8
9
8
10
11
9
9
11
7
9
9
13
16
12
13
11
12
12
6
10
10
9
10
10
13
11
13
14
8
12
8
10
8
11
14
9
11
7
12
14
12
11
8
9
9
6
6
14
12
14
15
14
9
10
11
11
15
11
11
12
10
16
6
13
17
12
9
11
12
11
13
10
12
10
6
8
9
12
11
10
11
8
9
9
10
9
9
10
14
11
10
8
12
15
14
7
13
8
10
13
8
15
9
8
9
8
7
11
18
13
8
9
8
9
8
12
8
9
10
9
11
9
15
10
9
10
10
11
6
12
8
11
11
9
12
8
9
10
9
7
13
8
10
8
11
7
10
11
10
8
11
10
8
11
9
13
11
7
9
11
8
13
12
10
13
9
9
12
7
10
8
10
15
13
11
8
10
10
12
9
7
8
9
9
7
7
7
14
9
9
12
10
11
10
17
10
10
10
10
7
13
11
10
8
10
10
9
13
10
9
14
10
9
11
12
11
7
9
13
17
8
9
7
9
7
7
11
6
11
17
9
18
10
10
10
12
12
17
9
10
10
8
9
13
9
11
11
11
10
9
14
8
11
8
8
11
10
10
11
7
11
9
11
12
8
8
13
7
9
8
11
12
14
10
11
10
10
8
13
13
9
11
11
9
17
10
9
9
10
9
10
13
10
11
9
6
11
8
14
12
8
14
12
11
10
9
13
8
11
10
9
7
10
9
7
9
9
8
7
8
10
9
11
11
6
8
12
9
10
11
10
18
9
12
8
9
10
11
12
16
6
8
8
8
11
11
9
15
11
15
9
7
10
9
8
11
11
9
10
12
12
10
9
9
9
15
6
6
16
10
7
10
6
10
10
10
10
11
11
12
9
9
10
10
10
6
8
11
13
9
9
9
10
9
11
15
13
14
17
9
11
10
7
8
10
9
9
9
10
10
6
10
12
12
8
11
11
8
11
13
7
14
9
10
10
12
10
8
10
11
9
10
6
10
14
15
12
7
11
10
18
14
13
9
13
10
8
13
12
11
9
11
9
10
12
8
7
8
9
10
11
13
13
7
18
11
9
11
11
10
8
12
16
10
12
14
10
7
10
10
8
11
10
6
9
10
10
7
12
10
8
18
13
8
15
7
11
7
11
14
11
10
8
15
11
9
11
10
8
12
10
12
6
14
10
10
10
12
8
11
7
9
8
10
7
7
10
9
10
11
7
9
8
12
10
7
7
15
19
15
11
8
9
9
12
11
12
7
13
10
9
7
7
8
10
11
12
8
10
12
12
9
9
12
8
13
12
15
14
12
15
18
10
11
8
12
11
9
8
12
7
11
9
8
17
8
11
13
12
8
14
10
15
10
12
11
11
9
10
8
18
12
13
9
11
11
10
10
12
11
10
9
7
14
9
10
15
14
9
12
11
12
8
10
9
8
8
11
9
11
10
12
11
9
19
10
18
12
10
9
13
9
7
14
7
9
12
11
10
11
12
11
17
16
8
14
8
7
7
10
11
9
11
11
9
10
12
10
8
10
8
12
11
8
10
8
6
8
11
9
13
12
11
19
12
6
11
10
15
11
18
14
9
10
14
7
11
9
12
19
12
10
8
12
9
10
9
8
8
11
8
12
11
15
7
11
10
9
13
15
10
8
10
14
10
10
13
13
7
10
11
10
10
12
12
8
7
12
11
8
8
12
9
11
13
6
9
9
12
12
13
8
8
13
6
8
10
13
15
10
12
6
9
13
10
9
10
13
9
12
6
8
9
8
6
9
7
11
12
9
10
13
11
12
10
10
15
12
12
6
9
9
10
10
13
8
13
10
16
14
11
7
9
13
8
10
8
10
9
9
10
7
9
8
9
9
10
11
15
8
11
8
6
13
14
11
16
8
12
8
11
7
7
9
15
13
9
11
14
16
11
7
12
14
10
8
10
6
11
9
10
15
11
8
10
8
11
9
11
7
12
10
11
10
7
11
6
9
15
7
11
8
16
19
9
10
7
8
7
10
6
8
9
9
12
9
11
8
8
10
11
10
12
11
9
8
12
11
8
13
9
9
10
13
12
9
12
9
9
8
8
8
9
19
13
11
7
13
7
10
11
11
11
13
6
12
10
11
10
7
11
9
9
12
9
12
9
11
8
12
11
9
10
10
10
13
12
8
8
10
12
15
18
12
15
9
12
12
9
12
6
13
10
10
10
12
11
13
15
6
9
6
10
12
15
7
9
9
9
7
8
9
10
7
9
12
7
8
7
9
12
9
11
10
9
7
13
9
8
11
8
14
17
11
10
10
11
9
8
16
13
7
10
9
11
10
8
10
8
9
9
10
11
11
12
7
9
11
8
7
11
11
11
7
10
9
13
10
12
11
10
13
8
18
12
11
16
10
9
9
12
12
10
11
10
9
11
8
10
11
12
8
13
10
14
9
10
10
10
10
14
12
9
13
11
9
10
12
7
6
9
14
16
8
13
6
10
8
8
12
15
10
11
12
10
12
10
10
14
10
11
6
8
9
10
8
10
11
11
14
14
8
12
10
12
6
12
11
14
10
11
9
10
10
13
16
10
16
8
17
10
11
13
10
7
11
9
11
15
9
14
9
16
8
9
9
12
10
13
11
14
10
13
10
11
13
13
12
9
13
12
16
18
10
11
13
17
13
11
11
9
13
10
9
15
7
8
14
12
9
11
10
12
11
11
8
12
10
14
10
9
7
9
7
9
11
11
10
8
9
14
11
12
11
13
12
17
13
7
10
11
9
10
11
9
7
11
12
8
12
11
8
8
12
9
11
11
10
7
9
10
8
12
8
13
8
11
13
11
11
9
11
13
13
11
9
9
11
9
9
7
11
8
10
10
10
9
7
12
10
13
13
8
10
8
12
9
11
19
11
9
8
7
12
9
16
12
10
7
13
14
11
9
16
14
10
12
8
11
8
7
12
6
11
10
11
15
9
10
11
9
11
13
10
9
13
11
13
8
10
8
8
12
9
11
11
11
9
7
14
12
14
9
9
14
9
10
9
8
9
10
10
8
9
6
12
12
7
11
9
9
13
15
11
8
8
8
8
10
13
12
9
9
11
9
12
11
10
9
9
11
12
9
19
9
11
9
11
11
14
10
8
12
7
14
12
12
11
10
10
14
12
9
13
13
11
9
11
11
7
14
9
12
13
12
7
11
10
10
7
11
8
14
9
13
6
11
10
9
14
16
6
10
17
9
13
8
7
10
11
8
13
8
18
14
8
11
13
12
11
10
11
12
7
9
11
9
8
10
8
6
11
8
7
9
8
14
13
18
11
11
10
13
15
8
6
8
8
13
7
11
11
9
10
7
10
8
13
12
9
11
8
7
8
16
12
14
7
14
10
9
11
9
12
14
8
13
11
14
9
12
12
11
8
11
11
8
7
7
8
10
14
9
9
9
12
11
13
12
12
7
12
7
10
9
12
8
12
10
10
11
8
13
7
14
10
12
13
8
10
8
16
8
12
7
7
10
18
12
11
7
10
10
8
12
11
11
10
10
12
9
11
6
10
12
9
13
8
15
13
10
11
17
9
8
9
8
12
15
7
10
10
11
10
12
14
12
6
7
10
8
11
9
10
11
12
11
11
12
11
12
12
7
11
6
11
10
10
9
12
11
10
9
11
14
11
8
10
15
10
7
13
9
12
9
11
6
9
10
7
10
11
12
11
8
11
12
12
7
12
12
10
9
9
8
6
8
9
13
11
8
11
14
10
18
14
14
9
7
12
9
14
7
13
16
15
12
7
13
13
8
7
9
13
14
11
10
10
11
13
9
10
11
10
11
16
11
11
12
11
11
15
13
12
9
7
7
17
11
14
18
10
14
15
9
12
6
7
9
10
14
11
9
7
6
9
12
6
14
10
14
7
12
7
10
8
8
12
10
6
12
11
10
10
6
11
7
13
9
10
11
14
9
11
15
13
10
11
9
13
11
7
8
17
14
17
15
17
8
6
9
12
10
10
11
17
10
15
12
15
10
9
9
13
10
14
18
12
8
11
10
11
8
9
8
7
12
15
8
10
7
13
9
9
11
10
10
8
15
8
8
10
17
14
14
18
10
11
10
12
7
10
11
8
10
10
10
10
9
11
8
9
7
16
11
11
12
10
14
11
13
6
10
14
8
7
11
12
10
14
13
10
8
6
11
14
9
13
13
13
8
16
9
6
13
10
11
11
6
11
13
10
9
17
9
11
9
11
7
17
10
13
12
7
9
10
6
13
10
10
8
7
11
7
12
14
11
10
7
10
9
10
10
8
9
8
10
11
9
16
11
14
11
8
9
7
6
7
10
8
12
13
7
13
12
9
6
8
9
16
12
9
11
12
12
10
6
12
14
8
15
9
9
15
6
7
15
10
6
7
7
9
7
13
13
12
14
9
10
10
13
10
13
10
11
13
10
7
8
16
14
17
8
13
9
11
9
10
14
10
10
9
10
14
11
9
7
10
11
14
13
13
13
13
8
8
7
11
15
11
11
10
10
10
9
11
9
9
19
12
14
13
10
14
10
10
7
12
6
11
10
12
11
11
14
10
11
10
17
13
13
12
12
9
11
19
9
11
8
11
9
13
15
10
13
10
8
13
12
12
8
9
6
10
7
6
9
10
6
9
13
9
15
11
14
14
8
10
10
12
9
11
10
14
12
9
8
9
6
8
9
12
10
7
13
6
9
9
9
13
10
7
11
9
9
9
12
12
13
11
9
7
11
11
15
10
7
7
11
13
11
10
11
8
17
12
7
8
10
7
10
8
12
14
6
8
10
12
10
7
9
8
7
8
12
8
14
11
7
15
15
7
13
9
8
12
7
9
8
11
9
11
11
8
6
8
8
9
10
7
14
10
14
16
13
8
10
8
6
13
10
6
15
10
11
9
9
7
10
8
9
10
11
10
9
13
15
9
14
11
19
13
9
9
10
13
9
10
10
9
7
10
8
9
9
15
12
12
12
10
7
9
9
12
8
8
13
9
17
16
13
11
11
15
12
7
10
7
10
7
10
12
8
17
10
11
8
14
12
14
12
10
11
10
12
10
9
12
9
15
9
10
9
10
13
10
10
8
7
11
11
10
14
13
9
7
15
13
10
8
10
8
14
8
14
9
16
8
11
12
7
12
9
12
12
11
6
10
10
10
15
7
9
12
11
13
9
8
11
8
16
12
11
9
10
7
16
11
14
9
10
10
10
13
10
13
11
6
10
11
10
7
10
13
10
11
7
6
12
10
8
9
10
10
9
12
11
13
9
12
12
10
11
8
11
14
13
13
11
12
8
6
12
9
8
10
11
9
12
13
6
7
13
14
10
10
11
12
8
12
10
10
9
11
7
8
12
14
12
8
7
11
13
9
10
11
13
12
9
12
16
7
9
10
12
7
12
14
7
12
9
9
17
8
11
7
12
14
10
12
18
13
16
13
11
10
9
9
11
13
13
12
6
11
13
11
7
8
8
7
11
11
11
12
10
10
9
13
9
7
10
10
15
10
8
13
11
13
6
12
8
7
8
10
8
16
13
14
8
10
6
11
12
10
10
16
8
13
11
12
7
15
9
9
6
7
10
12
12
12
10
18
13
14
17
9
11
12
11
6
9
6
12
7
13
6
10
9
9
9
9
13
8
10
13
12
14
12
10
11
9
14
10
18
11
9
7
8
12
9
8
10
11
11
10
10
10
13
11
8
10
10
9
14
10
11
15
11
9
8
12
16
11
11
12
10
9
15
11
7
12
11
9
13
6
8
12
10
11
10
12
11
9
16
10
9
13
12
12
10
11
8
9
16
14
10
8
8
10
13
8
12
9
8
10
16
7
11
11
11
13
9
8
6
8
8
16
12
18
6
13
7
13
10
13
10
11
11
11
10
6
6
6
9
11
6
12
12
12
11
13
7
10
10
11
7
13
11
11
10
11
7
8
10
12
10
9
12
8
12
8
10
11
10
7
12
12
11
9
8
7
13
6
16
9
11
8
12
15
10
12
8
9
15
12
8
11
8
12
8
9
11
10
11
7
11
6
11
10
13
17
15
10
9
12
9
8
10
9
9
13
7
10
10
11
8
6
14
10
9
12
19
10
8
7
13
9
8
13
9
11
8
14
9
14
13
11
11
12
9
16
12
15
11
15
10
10
15
8
15
8
12
7
7
11
9
10
14
10
13
14
11
10
8
15
16
12
11
10
9
10
9
14
16
12
13
7
27
15
11
8
18
11
9
18
7
11
15
18
17
10
14
13
9
6
7
14
8
7
14
12
8
8
10
10
16
10
11
15
12
9
12
18
11
8
12
14
8
11
15
19
10
17
8
14
9
12
11
8
11
9
13
6
8
11
11
7
11
10
15
13
8
14
12
9
13
8
8
14
14
11
14
6
7
10
10
13
10
9
10
6
13
11
11
15
11
8
9
17
11
14
14
9
7
15
14
10
8
13
8
9
10
11
9
8
13
9
9
15
11
9
6
14
10
9
10
11
11
10
8
9
7
10
11
11
13
10
10
8
6
8
12
11
8
8
14
13
8
9
11
13
6
10
14
11
9
12
9
7
10
9
11
13
8
12
12
8
10
12
8
14
13
9
9
9
16
12
10
11
11
10
12
12
9
11
12
8
10
11
8
18
11
7
10
9
13
13
13
11
7
11
9
6
6
15
12
12
8
15
13
13
8
13
10
11
9
11
14
10
8
6
8
11
10
13
10
18
11
13
10
12
9
11
10
9
11
19
19
9
12
12
12
11
11
7
19
12
7
7
11
6
11
8
9
8
11
10
7
12
8
10
15
6
12
14
12
7
9
7
14
10
12
10
13
10
8
9
8
14
12
19
6
6
7
14
11
6
11
12
9
12
13
8
10
14
8
9
11
15
8
10
14
8
8
11
14
8
6
8
11
11
11
8
8
11
10
10
6
10
13
11
13
9
10
8
7
9
12
14
10
11
9
11
11
13
10
9
7
10
6
11
13
10
10
9
10
9
9
11
9
8
8
12
9
13
15
11
11
12
6
14
8
8
7
18
11
8
11
8
7
9
8
7
10
9
16
12
11
10
15
10
8
11
13
9
8
10
10
11
11
8
11
11
10
16
9
10
8
16
10
10
8
7
8
17
12
9
11
8
7
9
11
10
7
11
14
14
10
9
9
14
8
13
12
8
13
12
6
6
14
14
8
17
10
8
13
7
14
9
12
13
18
11
10
13
8
18
11
14
16
11
12
11
18
6
14
12
10
11
10
6
13
13
9
9
14
9
16
12
9
15
8
12
11
16
12
7
11
12
12
15
9
10
14
9
10
11
9
10
9
11
14
7
10
13
11
9
10
9
16
10
13
10
8
15
8
9
15
16
16
9
8
9
12
8
12
10
8
13
16
11
8
18
9
11
14
9
16
9
9
9
14
10
11
13
9
12
8
14
9
13
10
11
9
9
11
11
8
9
8
14
14
10
9
7
10
9
10
11
12
13
15
10
8
9
11
9
13
12
8
12
8
9
12
10
10
11
6
9
8
12
16
9
14
8
11
16
13
14
10
8
14
12
14
12
13
17
8
16
11
13
12
9
11
14
15
6
9
12
7
7
12
15
11
9
15
9
9
10
9
11
12
9
10
12
16
9
13
10
8
8
7
12
11
9
13
13
6
12
13
14
8
10
11
7
9
10
10
11
8
12
8
14
13
14
15
9
18
8
12
13
9
11
13
14
6
15
11
13
14
14
9
10
6
11
14
12
11
10
14
14
12
9
12
12
12
7
12
16
7
7
11
11
10
10
6
6
8
10
14
16
7
9
10
16
11
12
12
14
7
8
15
10
12
10
6
11
14
9
10
13
12
8
9
9
10
8
9
11
8
8
12
8
9
6
8
10
16
8
14
12
9
17
11
11
14
10
9
11
10
8
6
11
14
10
9
9
6
10
11
13
12
15
14
16
12
9
9
9
9
9
11
10
8
12
9
9
14
10
9
10
13
11
10
9
11
6
8
10
12
11
11
10
12
13
12
9
8
11
9
11
10
14
12
11
9
11
8
10
13
8
10
7
11
9
10
10
13
10
9
12
10
16
18
12
8
11
9
8
10
13
13
7
14
10
10
11
9
9
11
11
12
9
11
14
9
8
11
11
13
9
7
9
14
13
8
13
15
12
10
6
10
13
11
10
14
10
17
15
8
8
12
10
9
10
15
11
11
16
9
12
11
15
15
13
16
10
9
9
7
11
10
10
9
10
10
10
11
10
9
11
10
11
8
16
9
10
8
8
7
11
8
12
8
12
11
9
19
15
11
13
11
13
10
11
15
7
11
8
8
10
8
13
13
11
9
9
9
11
9
8
8
14
12
11
14
13
10
12
9
12
9
10
17
10
8
7
8
13
9
13
11
12
11
6
14
13
12
18
16
9
13
12
6
10
9
10
7
10
9
19
18
9
10
15
15
10
12
10
9
13
11
8
9
11
9
11
12
10
10
8
6
16
8
11
10
19
15
15
15
9
13
11
12
11
9
12
9
10
9
11
13
9
10
10
7
11
9
9
13
13
7
10
11
15
10
9
14
10
15
12
12
10
6
11
12
17
12
9
7
10
7
10
10
12
9
10
10
15
14
11
10
10
9
8
9
13
11
8
10
12
9
10
11
11
14
12
6
10
9
12
12
14
12
8
14
9
11
9
7
9
12
14
6
11
11
11
13
10
13
9
9
10
14
15
14
6
6
9
8
12
8
8
11
7
11
15
8
10
8
15
11
15
11
9
10
13
11
9
10
13
6
12
11
10
18
14
11
12
7
7
12
6
10
12
8
10
12
9
17
12
11
8
15
7
19
9
8
7
13
13
10
9
15
12
10
12
9
14
14
7
11
10
11
11
11
8
13
6
11
19
10
10
8
15
9
6
7
8
15
11
13
14
14
9
12
9
15
14
10
11
10
9
12
13
9
8
10
11
17
11
12
11
10
8
10
10
6
11
9
6
10
7
9
6
6
10
11
11
19
11
13
15
8
10
11
9
15
14
7
9
8
11
13
11
10
16
18
10
6
10
11
14
9
10
10
14
11
11
8
14
13
11
8
11
7
19
9
11
9
10
13
8
10
8
13
12
10
7
9
9
19
13
13
15
12
10
11
16
8
11
14
9
11
12
13
10
13
9
12
9
7
6
12
12
11
11
15
11
9
9
10
8
10
10
7
7
9
10
13
12
9
9
10
11
12
10
11
14
12
14
15
7
8
9
11
7
11
7
13
13
8
7
11
10
11
13
11
8
9
9
9
8
6
13
10
13
13
7
10
7
12
11
11
11
7
10
8
15
7
6
11
12
11
10
12
7
15
12
8
12
8
6
13
17
7
9
11
9
12
12
13
15
8
9
9
9
7
8
18
13
13
10
12
11
11
13
13
16
12
6
7
11
9
14
14
10
11
8
9
9
15
6
9
8
14
8
11
13
11
12
10
10
11
17
14
11
13
8
19
11
10
7
11
7
8
13
11
10
10
13
10
10
11
14
6
10
9
16
7
6
10
12
8
12
8
19
9
7
10
15
11
8
10
10
11
9
13
13
9
11
11
13
12
8
9
14
14
9
8
12
13
12
12
12
9
13
7
10
13
9
15
12
9
9
9
10
10
7
13
11
11
9
13
13
9
17
6
14
8
13
12
12
12
17
12
10
14
11
12
10
10
13
13
9
7
11
12
9
13
12
9
8
11
11
11
11
8
10
16
15
14
12
10
8
11
7
11
19
11
8
7
14
9
8
19
18
6
8
11
13
7
14
12
9
12
6
15
10
11
13
9
9
11
17
9
12
10
12
17
12
11
17
6
10
14
10
14
14
10
16
9
11
15
8
9
10
9
11
9
11
10
9
12
11
17
14
8
6
11
10
8
13
9
8
15
12
7
10
10
8
10
9
14
13
11
10
8
8
9
11
14
10
8
15
11
8
13
11
10
11
8
9
10
8
6
18
7
8
16
10
9
12
10
10
13
9
8
9
12
15
13
11
9
9
11
11
10
9
11
9
8
9
19
6
10
14
10
9
7
8
15
10
12
19
8
11
9
10
6
11
9
10
9
13
9
11
6
8
18
9
12
15
8
9
14
7
11
9
11
13
7
15
13
14
9
17
11
12
14
13
7
10
11
13
10
7
12
16
10
7
9
6
12
13
7
6
10
7
7
8
13
11
11
9
7
19
11
12
10
15
11
14
14
6
8
10
7
12
8
8
13
10
10
11
10
18
13
12
10
10
13
12
9
8
12
13
11
9
10
11
9
10
10
8
11
13
16
13
10
12
10
11
10
13
12
10
15
11
9
11
19
14
12
7
7
8
6
14
12
6
7
9
15
13
8
10
11
14
15
13
11
10
17
17
12
7
14
10
7
14
13
18
10
10
12
10
13
7
11
8
11
6
8
6
11
13
18
9
15
10
13
6
14
8
10
12
12
7
10
8
11
9
11
14
13
12
17
9
12
10
8
13
13
11
11
11
11
10
18
15
9
8
8
11
8
10
10
12
12
11
8
12
10
9
12
9
9
6
10
12
9
10
13
9
11
11
16
12
18
14
8
14
14
11
10
15
16
9
8
8
9
8
12
10
12
11
8
7
11
13
13
11
18
14
11
10
11
10
11
14
9
9
6
7
10
13
8
11
16
11
14
17
13
12
7
10
9
12
16
12
12
8
10
10
10
9
11
7
9
14
7
9
13
14
10
10
8
13
16
12
9
13
9
9
11
10
15
12
13
13
10
10
17
12
12
11
11
9
10
11
13
17
9
14
11
10
9
17
11
9
9
12
12
8
15
10
11
12
12
10
10
8
12
9
9
11
11
11
8
12
10
12
12
8
10
11
12
13
18
11
9
10
11
13
10
9
11
8
14
9
9
16
10
15
18
17
6
15
15
16
13
8
6
10
9
8
12
10
11
12
6
14
14
7
12
18
6
7
15
9
11
9
6
8
11
11
10
10
18
13
9
11
19
7
11
11
7
9
6
12
9
12
15
13
14
13
11
16
7
11
9
10
13
9
12
11
17
18
10
6
14
12
6
18
12
9
8
11
12
7
9
10
7
9
8
13
19
11
8
9
15
9
10
8
14
9
9
7
10
10
16
9
12
9
7
11
11
16
12
12
10
18
10
7
8
12
9
13
11
16
12
9
11
11
10
11
12
12
15
12
9
11
11
14
8
12
11
12
10
13
8
12
11
11
13
14
15
13
11
7
13
9
10
15
17
11
9
9
8
10
11
11
11
10
7
8
6
10
8
11
15
9
10
15
6
6
11
14
10
9
9
10
9
11
9
10
10
8
10
11
6
8
12
10
11
6
11
16
8
17
10
12
9
15
6
14
9
10
8
9
11
10
11
12
14
11
9
10
11
8
9
7
10
7
14
11
17
9
10
8
11
10
12
9
6
15
11
11
9
9
12
9
9
9
9
10
9
16
6
14
13
10
10
14
12
15
17
10
10
11
10
12
14
9
11
6
15
12
19
18
10
12
10
13
6
9
11
12
8
14
13
17
7
10
11
8
8
17
10
11
14
7
11
12
8
16
7
18
11
9
13
18
14
16
15
11
9
11
11
13
8
12
8
8
10
11
12
9
15
8
13
14
18
11
8
10
10
10
13
10
13
10
10
8
11
10
9
7
11
9
11
9
8
11
15
11
7
11
12
12
11
8
13
16
16
9
17
10
15
9
13
11
18
12
9
9
10
9
12
12
7
17
13
8
11
11
7
10
8
11
8
14
10
11
12
7
9
8
11
11
12
9
16
8
8
12
12
7
8
10
6
10
13
14
11
8
11
9
11
10
10
14
16
18
10
7
13
10
11
9
12
12
10
14
15
10
14
11
8
12
13
6
11
11
17
13
17
10
9
14
10
13
11
11
13
7
10
12
13
17
10
8
10
10
9
13
10
7
11
11
12
6
10
8
14
9
6
15
11
15
12
14
13
13
7
8
10
11
8
9
12
15
11
10
11
9
8
13
15
10
9
13
13
13
15
10
14
10
10
10
8
7
6
7
9
16
11
12
9
19
8
14
13
7
7
8
13
15
10
9
11
15
12
15
14
19
12
9
12
6
15
15
7
11
8
15
7
9
12
13
12
8
11
12
9
11
15
13
19
12
12
16
12
12
11
8
20
12
14
8
9
10
9
17
10
16
11
9
11
7
12
11
8
8
15
9
18
13
11
10
12
16
16
16
6
10
7
17
11
8
9
8
14
11
13
13
16
8
16
9
7
16
11
10
8
8
9
8
9
9
17
10
11
13
17
8
7
14
10
7
10
10
15
11
10
10
9
13
9
13
8
10
8
8
15
15
14
9
6
8
10
7
8
13
11
12
11
16
7
13
9
10
11
16
18
9
13
6
13
8
10
10
8
11
8
7
9
8
10
11
13
10
10
10
11
10
8
12
13
10
15
15
11
11
14
9
13
9
11
10
9
14
8
7
12
19
12
10
8
13
12
9
13
18
10
17
14
11
14
6
14
7
13
6
9
10
15
6
8
10
11
13
17
10
11
8
12
11
13
7
14
18
18
10
8
10
10
12
16
10
7
10
12
8
17
8
11
15
6
9
13
10
14
9
7
11
6
12
10
12
13
8
11
13
11
10
10
11
9
13
7
14
6
17
8
8
8
9
11
10
12
8
11
7
18
8
9
7
12
11
10
6
18
6
10
13
11
12
11
9
13
14
10
13
12
8
10
9
11
12
9
12
13
12
7
9
7
10
11
12
9
16
9
16
8
12
11
10
11
13
12
14
17
9
10
7
11
11
12
14
10
10
12
18
8
19
9
11
11
10
15
12
10
12
15
6
11
13
10
11
9
7
7
12
8
6
12
18
11
7
10
11
11
11
6
10
12
10
11
12
10
10
12
6
18
14
13
10
11
15
14
15
13
10
10
18
9
10
10
10
9
11
15
11
10
7
7
13
17
8
14
11
12
13
10
15
10
10
7
8
12
9
8
15
13
14
9
11
19
11
10
16
12
8
9
10
12
11
9
8
11
12
7
13
12
11
13
9
15
19
8
13
8
11
9
11
9
17
9
11
14
11
8
10
7
10
17
8
9
8
11
11
6
15
12
10
6
9
9
15
10
10
19
10
18
10
14
7
12
12
14
7
8
14
10
8
13
11
15
10
13
6
12
13
13
12
6
13
7
13
12
11
19
14
11
14
13
8
9
7
8
14
12
14
6
7
18
14
7
11
9
12
19
8
12
13
10
9
18
14
15
9
7
15
10
10
9
11
13
10
13
11
8
7
8
17
11
16
10
9
18
14
12
13
9
11
11
10
10
11
10
12
9
11
11
13
11
9
14
8
11
12
10
13
9
10
10
8
9
15
6
12
10
6
10
10
11
19
12
8
11
12
8
9
12
10
13
9
7
11
10
8
12
10
18
18
8
15
11
11
14
11
14
8
12
9
10
11
12
15
10
11
17
10
8
15
16
12
13
12
12
15
13
21
12
17
7
8
8
11
9
8
10
15
12
11
17
10
10
15
11
11
9
9
17
16
9
9
9
17
6
12
7
9
9
8
12
8
18
10
8
10
11
9
13
10
11
13
16
13
14
11
13
14
10
12
6
13
11
10
11
9
9
9
13
9
6
14
13
8
6
9
11
8
9
10
14
11
7
11
16
13
13
14
17
13
8
6
13
10
10
11
8
15
12
10
6
10
12
10
12
8
11
12
9
10
11
9
13
12
10
10
13
11
14
12
12
7
11
14
7
19
11
10
6
7
11
11
11
13
12
8
10
10
8
14
13
9
13
9
13
12
12
11
8
9
10
8
11
14
8
11
10
19
12
8
10
9
11
7
14
6
8
12
9
12
10
8
18
11
8
11
10
18
13
14
18
14
7
12
9
10
8
18
11
10
14
11
14
12
7
8
10
11
14
8
11
10
14
8
19
11
8
9
11
9
7
10
10
7
15
8
11
8
8
7
19
9
12
15
9
7
6
17
10
7
9
10
11
12
8
12
14
15
10
9
11
13
8
8
16
11
14
12
12
9
13
8
10
16
8
12
14
12
13
15
11
7
6
18
11
8
11
8
6
8
14
10
11
10
10
12
10
6
6
12
7
9
10
7
9
9
11
14
9
9
14
14
10
7
7
11
9
9
10
10
7
9
11
10
9
12
10
9
6
12
10
14
6
18
14
12
13
11
14
8
11
12
13
10
11
19
8
10
7
7
8
9
13
10
12
16
11
13
6
10
10
8
14
8
10
13
13
9
8
15
8
18
12
11
13
8
16
12
12
11
12
10
11
12
7
8
13
6
12
16
18
7
10
9
8
9
10
11
17
8
11
11
10
13
14
8
6
9
11
6
15
12
10
9
11
16
9
11
10
9
10
14
13
13
12
13
9
12
11
7
7
8
10
8
14
8
10
9
12
12
7
9
9
10
14
10
18
10
15
12
6
11
18
13
11
10
14
11
11
8
17
6
15
16
15
9
10
7
10
10
10
11
14
9
12
8
17
14
9
8
7
13
14
10
10
17
7
11
6
14
13
8
11
11
8
8
17
12
11
14
13
7
8
11
8
7
12
10
17
11
11
11
8
17
12
8
16
7
11
10
12
7
9
7
12
12
14
11
12
8
17
15
12
10
15
13
17
12
9
9
8
6
12
11
10
7
10
9
10
11
13
10
17
13
12
8
12
15
11
14
11
9
9
13
10
14
8
7
13
14
13
10
13
9
8
10
18
13
9
9
13
7
11
10
12
8
8
10
15
17
14
9
12
13
7
12
9
11
17
11
12
18
10
16
10
9
16
10
15
w = pd.read_csv('/Users/glebsokolov/projects/metadata.tsv')
import plotly.express as px
d = pd.concat([w.astype(str), pd.DataFrame.from_records(reduced_pca),pd.DataFrame.from_records(reduced_lle) ], axis=1).iloc[:1000,:]
d.columns = ['word', 'pca1', 'pca2', 'lle1', 'lle2']
d.word = d.word.apply(lambda x: x[2:-1])
d
| word | pca1 | pca2 | lle1 | lle2 | |
|---|---|---|---|---|---|
| 0 | the | -0.060 | -0.066 | -0.001 | -0.002 |
| 1 | a | 0.019 | -0.078 | 0.001 | -0.008 |
| 2 | i | -0.049 | -0.040 | -0.006 | -0.004 |
| 3 | to | -0.060 | 0.023 | -0.016 | -0.011 |
| 4 | and | -0.087 | -0.011 | -0.009 | -0.006 |
| ... | ... | ... | ... | ... | ... |
| 995 | murder | -0.237 | -0.094 | -0.014 | -0.014 |
| 996 | thick | 0.077 | -0.048 | 0.002 | -0.006 |
| 997 | kinda | -0.013 | 0.080 | -0.006 | 0.004 |
| 998 | john | -0.030 | -0.062 | -0.002 | -0.003 |
| 999 | national | -0.228 | 0.049 | -0.014 | -0.002 |
1000 rows × 5 columns
px.scatter(data_frame=d, x='pca1', y='pca2', hover_name= 'word')
px.scatter(data_frame=d, x='lle1', y='lle2', hover_name= 'word')
%load_ext tensorboard
%tensorboard --logdir log_dir
import os
save_path = os.path.join(os.getcwd(), 'hateSpeechDetectionModelAdjusted.h5')
model.save(save_path) # saved after learning, load these weights in future
# model.load_weights(save_path)
The Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details.
from keras.preprocessing.sequence import pad_sequences
def convert_sentence(sentence):
sentence = tf.strings.substr(sentence, 0, 300)
sentence = tf.strings.regex_replace(sentence, b"<br\\s*/?>", b" ")
sentence = tf.strings.regex_replace(sentence, b"[^a-zA-Z']", b" ")
sentence = tf.strings.split(sentence)
sentence = table.lookup(sentence)
return pad_sequences([sentence], 34)
def make_prediction(sentence):
representation = convert_sentence(sentence)
d = {0:'hate speech', 1:'offensive language', 2:'neutral'}
pred = model.predict(representation)
res = dict(zip(pred.reshape(3, ), d.values()))
print(f'Mostly likely this is {colored(d[np.argmax(pred)], attrs=["bold"])}; probabilities for each of classes are:')
for i in res.items():
print(f'{i[0]*100:.3f}%: {i[1]}')
t = [
"fuck you fucking fuck",
"i hate them",
"you can take that pus-fucking nigger, enjoy watching him eat your stinking head off and your hairy slit, but I suggest you stop fucking around with that ungrateful chink, you fucker, go back to your monogamous land and take him with you, you're just a faggot for once a week to fuck, cause you're a fucking squeaky moaning twink. good luck in this happy marriage of hammerhead shark and muskrat.",
]
for sentence in t:
print(f'The sentence: "{sentence}"')
make_prediction(sentence)
print('\n')
The sentence: "fuck you fucking fuck" Mostly likely this is offensive language; probabilities for each of classes are: 34.076%: hate speech 65.729%: offensive language 0.194%: neutral The sentence: "i hate them" Mostly likely this is hate speech; probabilities for each of classes are: 67.794%: hate speech 26.344%: offensive language 5.862%: neutral The sentence: "you can take that pus-fucking nigger, enjoy watching him eat your stinking head off and your hairy slit, but I suggest you stop fucking around with that ungrateful chink, you fucker, go back to your monogamous land and take him with you, you're just a faggot for once a week to fuck, cause you're a fucking squeaky moaning twink. good luck in this happy marriage of hammerhead shark and muskrat." Mostly likely this is offensive language; probabilities for each of classes are: 30.939%: hate speech 67.459%: offensive language 1.602%: neutral